import requests
from lxml import etree
import json

main_url = "https://www.bg60.cc/book/3311/"

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
    "Referer": "https://www.bg60.cc/book/3311/",
    "Cookie": "Hm_lvt_52624d0257fe48ed9dea61ff01fa3417=1721008461; HMACCOUNT=E845D6D95CED5122; Hm_lpvt_52624d0257fe48ed9dea61ff01fa3417=1721009559",
}


def download(data):
    for k, v in data.items():
        if v.startswith("http"):
            print(k, "开始下载")
            r = requests.get(url=v, headers=headers)
            tree = etree.HTML(r.text)
            title = tree.xpath("//h1/text()")[0]
            # 章节的内容
            text = "\n".join(tree.xpath('//div[@id="chaptercontent"]/text()'))
            # 下一章的链接
            with open("全职高手.txt", "a", encoding='utf-8') as f:
                f.write(title + "\n")
                f.write(text + "\n")
            print(f"{title}下载完成")


def index_request(url):
    r = requests.get(url=url, headers=headers)
    r = etree.HTML(r.text)
    title = r.xpath('//div[@class="listmain"]/dl/dd/a/text()')
    url = [
        "https://www.bg60.cc" + x
        for x in r.xpath('//div[@class="listmain"]/dl/dd/a/@href')
    ]
    # with open('index.json', "w", encoding="utf-8") as f:
    #     json.dump(dict(zip(title,url)), f, indent="  ", ensure_ascii=False)
    with open("index.json", "r", encoding='utf-8') as f:
        download(json.load(f))


index_request(main_url)
